
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;

public class Lexer
{
	public Collection<Token> process(String text) throws IOException
	{
		Collection<Token> tokens = new ArrayList<>();
		Scanner scanner = new Scanner(new StringReader(text));
		Token token = scanner.yylex();

        while(token != null)
        {
        	tokens.add(token);
            token = scanner.yylex();
        }

        return tokens;
	}
}
